{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一种改进的前馈神经网络算法实现\n",
    "\n",
    "改进包括增加的交叉熵的成本函数，正规化，更好的网络权重的初始化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# library\n",
    "import json\n",
    "import random\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "# 定义交叉熵代价函数\n",
    "class CrossEntropyCost(object):\n",
    "    \"\"\"用来表示交叉熵代价的类\"\"\"\n",
    "    \n",
    "    @staticmethod\n",
    "    def fn(a, y):\n",
    "        \"\"\"交叉熵代价函数\"\"\"\n",
    "        return np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a)))\n",
    "        # np.nan_to_num 调用确保了 Numpy 正确处理接近 0 的对数值\n",
    "        \n",
    "    @staticmethod\n",
    "    def delta(z, a, y):\n",
    "        \"\"\"网络输出误差\"\"\"\n",
    "        return (a-y)\n",
    "    \n",
    "# 定义二次代价函数\n",
    "class QuadraticCost(object):\n",
    "    \"\"\"二次代价函数\"\"\"\n",
    "    \n",
    "    @staticmethod\n",
    "    def fn(a, y):\n",
    "        \"\"\"二次代价函数\"\"\"\n",
    "        return 0.5 * np.linalg.norm(a-y)**2\n",
    "    \n",
    "    @staticmethod\n",
    "    def delta(z, a, y):\n",
    "        \"\"\"网络输出误差\"\"\"\n",
    "        return (a-y) * sigmoid_prime(z)\n",
    "\n",
    "# 前馈神经网络\n",
    "class Network(object):\n",
    "    def __init__(self, sizes, cost=CrossEntropyCost):\n",
    "        self.num_layers = len(sizes)\n",
    "        self.sizes = sizes\n",
    "        self.default_weight_initializer()\n",
    "        self.cost = cost\n",
    "        \n",
    "    def default_weight_initializer(self):\n",
    "        \"\"\"\n",
    "            新式改进后的初始化权重方法\n",
    "            权重初始化，使用了均值为 0 而标准差为 1/sqrt(n),n 为对应的输入连接个数\n",
    "            偏置与之前一样，使用均值为 0 标准差为 1 的高斯分布\n",
    "        \"\"\"\n",
    "        self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]\n",
    "        self.weights = [np.random.randn(y, x)/np.sqrt(x) for x,y in zip(self.sizes[:-1], self.sizes[1:])]\n",
    "        \n",
    "    def large_weight_initializer(self):\n",
    "        \"\"\"\n",
    "            最开始的初始化权重方法\n",
    "            权重和偏置，使用了均值为0，而标准差为1的高斯分布\n",
    "        \"\"\"\n",
    "        self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]\n",
    "        self.weights = [np.random.randn(y, x) for x, y in zip(self.sizes[:-1], self.sizes[1:])]\n",
    "        \n",
    "    def feedforward(self, a):\n",
    "        \"\"\"对与给定的输入``a``， 返回网络的输出\"\"\"\n",
    "        for b,w in zip(self.biases, self.weights):\n",
    "            a = sigmoid(np.dot(w, a) + b)\n",
    "        return a\n",
    "    \n",
    "    def SGD(self, training_data, epochs, mini_batch_size, eta, lmbda=0.0, evaluation_data=None, \n",
    "            monitor_evaluation_cost=False, monitor_evaluation_accuracy=False, \n",
    "            monitor_training_cost=False, monitor_training_accuracy=False):\n",
    "        \"\"\"使用 小批量随机梯度下降算法 训练神经网络\n",
    "        \n",
    "            返回包含4个list的tuple，分别是每次迭代的验证集的代价，验证集的精确率，训练集的代价，训练集的精确率\"\"\"\n",
    "        if evaluation_data: n_eval = len(evaluation_data)\n",
    "        n_train = len(training_data)\n",
    "        evaluation_cost, evaluation_accuracy = [], []\n",
    "        training_cost, training_accuracy = [], []\n",
    "        for j in range(epochs):\n",
    "            random.shuffle(training_data)\n",
    "            mini_batches = [training_data[k:k+mini_batch_size] for k in range(0, n_train, mini_batch_size)]\n",
    "            \n",
    "            for mini_batch in mini_batches:\n",
    "                self.update_mini_batch(mini_batch, eta, lmbda, len(training_data))\n",
    "            \n",
    "            print(\"Epoch %s training complete\" % j)\n",
    "            if monitor_training_cost:\n",
    "                cost = self.total_cost(training_data, lmbda)\n",
    "                training_cost.append(cost)\n",
    "                print(\"Cost on training data: {}\".format(cost))\n",
    "            if monitor_training_accuracy:\n",
    "                accuracy = self.accuracy(training_data, convert=True)\n",
    "                training_accuracy.append(accuracy)\n",
    "                print(\"Accuracy on training: {} / {}\".format(accuracy, n_train))\n",
    "            if monitor_evaluation_cost:\n",
    "                cost = self.total_cost(evaluation_data, lmbda, convert=True)\n",
    "                evaluation_cost.append(cost)\n",
    "                print(\"Cost on evaluation data: {}\".format(cost))\n",
    "            if monitor_evaluation_accuracy:\n",
    "                accuracy = self.accuracy(evaluation_data)\n",
    "                evaluation_accuracy.append(accuracy)\n",
    "                print(\"Accuracy on evaluation data: {} / {}\".format(accuracy, n_eval))\n",
    "                \n",
    "            print('\\n')\n",
    "        return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy\n",
    "    \n",
    "    def update_mini_batch(self, mini_batch, eta, lmbda, n):\n",
    "        \"\"\"更新网络的权重\"\"\"\n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        for x,y in mini_batch:\n",
    "            delta_nabla_b, delta_nabla_w = self.backprop(x,y)\n",
    "            nabla_b = [nb+dnb for nb,dnb in zip(nabla_b, delta_nabla_b)]\n",
    "            nabla_w = [nw+dnw for nw,dnw in zip(nabla_w, delta_nabla_w)]\n",
    "            \n",
    "        self.weights = [(1-eta*(lmbda/n))*w -(eta/len(mini_batch))*nw for w,nw in zip(self.weights, nabla_w)]\n",
    "        self.biases = [b-(eta/len(mini_batch))*nb for b,nb in zip(self.biases, nabla_b)]\n",
    "        \n",
    "    def backprop(self, x, y):\n",
    "        \"\"\"Return a tuple ``(nabla_b, nabla_w)`` representing the gradient for the cost function C_x.\"\"\"\n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        # feedforward\n",
    "        activation = x\n",
    "        activations = [x]\n",
    "        zs = []\n",
    "        for b,w in zip(self.biases, self.weights):\n",
    "            z = np.dot(w, activation) + b\n",
    "            zs.append(z)\n",
    "            activation = sigmoid(z)\n",
    "            activations.append(activation)\n",
    "        # backward pass\n",
    "        delta = (self.cost).delta(zs[-1], activations[-1], y)\n",
    "        nabla_b[-1] = delta\n",
    "        nabla_w[-1] = np.dot(delta, activations[-2].transpose())\n",
    "        for l in range(2, self.num_layers):\n",
    "            z = zs[-l]\n",
    "            sp = sigmoid_prime(z)\n",
    "            delta = np.dot(self.weights[-l+1].transpose(), delta) * sp\n",
    "            nabla_b[-l] = delta\n",
    "            nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())\n",
    "        return (nabla_b, nabla_w)\n",
    "    \n",
    "    def accuracy(self, data, convert=False):\n",
    "        if convert:\n",
    "            results = [(np.argmax(self.feedforward(x)), np.argmax(y)) for (x,y) in data]\n",
    "        else:\n",
    "            results = [(np.argmax(self.feedforward(x)), y) for (x,y) in data]\n",
    "        return sum(int(x==y) for (x,y) in results)\n",
    "    \n",
    "    def total_cost(self, data, lmbda, convert=False):\n",
    "        cost = 0.0\n",
    "        for x, y in data:\n",
    "            a = self.feedforward(x)\n",
    "            if convert: y = vectorized_result(y)\n",
    "            cost += self.cost.fn(a,y)/len(data)\n",
    "        cost += 0.5*(lmbda/len(data))*sum(np.linalg.norm(w)**2 for w in self.weights)\n",
    "        return cost\n",
    "    \n",
    "    def save(self, filename):\n",
    "        data = {\"sizes\": self.sizes,\n",
    "               \"weights\": [w.tolist() for w in self.weights],\n",
    "               \"biases\": [b.tolist() for b in self.biases],\n",
    "               \"cost\": str(self.cost.__name__)}\n",
    "        f = opne(filename, \"w\")\n",
    "        json.dump(data, f)\n",
    "        f.close()\n",
    "        \n",
    "def load(filename):\n",
    "    f = open(filename, \"r\")\n",
    "    data = json.load(f)\n",
    "    f.close()\n",
    "    cost = getattr(sys.modules[__name__], data[\"cost\"])\n",
    "    net = Network(data[\"sizes\"], cost=cost)\n",
    "    net.weights = [np.array(w) for w in data[\"weights\"]]\n",
    "    net.biases = [np.array(b) for b in data[\"biases\"]]\n",
    "    return net\n",
    "\n",
    "def vectorized_result(j):\n",
    "    e = np.zeros((10, 1))\n",
    "    e[j] = 1.0\n",
    "    return e\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1.0 / (1.0 + np.exp(-z))\n",
    "\n",
    "def sigmoid_prime(z):\n",
    "    return sigmoid(z) * (1-sigmoid(z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 加载数据集\n",
    "import mnist_loader\n",
    "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用交叉熵来对 MNIST 数字进行分类\n",
    "\n",
    "更之前一样，我们会使用一个包含 30 个隐藏元的网络,而小批量数据的大小也设置为 10。我们将学习速率设置为 η = 0.5 然后训练 30 个迭代期。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 training complete\n",
      "Accuracy on evaluation data: 9093 / 10000\n",
      "\n",
      "\n",
      "Epoch 1 training complete\n",
      "Accuracy on evaluation data: 9175 / 10000\n",
      "\n",
      "\n",
      "Epoch 2 training complete\n",
      "Accuracy on evaluation data: 9321 / 10000\n",
      "\n",
      "\n",
      "Epoch 3 training complete\n",
      "Accuracy on evaluation data: 9370 / 10000\n",
      "\n",
      "\n",
      "Epoch 4 training complete\n",
      "Accuracy on evaluation data: 9378 / 10000\n",
      "\n",
      "\n",
      "Epoch 5 training complete\n",
      "Accuracy on evaluation data: 9394 / 10000\n",
      "\n",
      "\n",
      "Epoch 6 training complete\n",
      "Accuracy on evaluation data: 9401 / 10000\n",
      "\n",
      "\n",
      "Epoch 7 training complete\n",
      "Accuracy on evaluation data: 9470 / 10000\n",
      "\n",
      "\n",
      "Epoch 8 training complete\n",
      "Accuracy on evaluation data: 9476 / 10000\n",
      "\n",
      "\n",
      "Epoch 9 training complete\n",
      "Accuracy on evaluation data: 9461 / 10000\n",
      "\n",
      "\n",
      "Epoch 10 training complete\n",
      "Accuracy on evaluation data: 9463 / 10000\n",
      "\n",
      "\n",
      "Epoch 11 training complete\n",
      "Accuracy on evaluation data: 9472 / 10000\n",
      "\n",
      "\n",
      "Epoch 12 training complete\n",
      "Accuracy on evaluation data: 9467 / 10000\n",
      "\n",
      "\n",
      "Epoch 13 training complete\n",
      "Accuracy on evaluation data: 9501 / 10000\n",
      "\n",
      "\n",
      "Epoch 14 training complete\n",
      "Accuracy on evaluation data: 9504 / 10000\n",
      "\n",
      "\n",
      "Epoch 15 training complete\n",
      "Accuracy on evaluation data: 9462 / 10000\n",
      "\n",
      "\n",
      "Epoch 16 training complete\n",
      "Accuracy on evaluation data: 9516 / 10000\n",
      "\n",
      "\n",
      "Epoch 17 training complete\n",
      "Accuracy on evaluation data: 9496 / 10000\n",
      "\n",
      "\n",
      "Epoch 18 training complete\n",
      "Accuracy on evaluation data: 9516 / 10000\n",
      "\n",
      "\n",
      "Epoch 19 training complete\n",
      "Accuracy on evaluation data: 9513 / 10000\n",
      "\n",
      "\n",
      "Epoch 20 training complete\n",
      "Accuracy on evaluation data: 9523 / 10000\n",
      "\n",
      "\n",
      "Epoch 21 training complete\n",
      "Accuracy on evaluation data: 9529 / 10000\n",
      "\n",
      "\n",
      "Epoch 22 training complete\n",
      "Accuracy on evaluation data: 9521 / 10000\n",
      "\n",
      "\n",
      "Epoch 23 training complete\n",
      "Accuracy on evaluation data: 9520 / 10000\n",
      "\n",
      "\n",
      "Epoch 24 training complete\n",
      "Accuracy on evaluation data: 9521 / 10000\n",
      "\n",
      "\n",
      "Epoch 25 training complete\n",
      "Accuracy on evaluation data: 9540 / 10000\n",
      "\n",
      "\n",
      "Epoch 26 training complete\n",
      "Accuracy on evaluation data: 9543 / 10000\n",
      "\n",
      "\n",
      "Epoch 27 training complete\n",
      "Accuracy on evaluation data: 9529 / 10000\n",
      "\n",
      "\n",
      "Epoch 28 training complete\n",
      "Accuracy on evaluation data: 9525 / 10000\n",
      "\n",
      "\n",
      "Epoch 29 training complete\n",
      "Accuracy on evaluation data: 9534 / 10000\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([],\n",
       " [9093,\n",
       "  9175,\n",
       "  9321,\n",
       "  9370,\n",
       "  9378,\n",
       "  9394,\n",
       "  9401,\n",
       "  9470,\n",
       "  9476,\n",
       "  9461,\n",
       "  9463,\n",
       "  9472,\n",
       "  9467,\n",
       "  9501,\n",
       "  9504,\n",
       "  9462,\n",
       "  9516,\n",
       "  9496,\n",
       "  9516,\n",
       "  9513,\n",
       "  9523,\n",
       "  9529,\n",
       "  9521,\n",
       "  9520,\n",
       "  9521,\n",
       "  9540,\n",
       "  9543,\n",
       "  9529,\n",
       "  9525,\n",
       "  9534],\n",
       " [],\n",
       " [])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = Network([784, 30, 10], cost=CrossEntropyCost)\n",
    "net.large_weight_initializer()\n",
    "net.SGD(training_data, 30, 10, 0.5, evaluation_data=test_data, monitor_evaluation_accuracy=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "运行上面的代码我们得到了一个 95.34% 准确率的网络。这跟我们之前使用二次代价函数得到的结果相当接近了,95.27%。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
